{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "431371d9-5587-433e-8e66-c801a3563e1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_cpp import Llama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7c98944d-7b6a-46a3-932e-113891948a4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: Tesla T4, compute capability 7.5, VMM: yes\n",
      "llama_model_loader: loaded meta data with 26 key-value pairs and 995 tensors from ./mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = ..\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 32768\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:                         llama.expert_count u32              = 8\n",
      "llama_model_loader: - kv  10:                    llama.expert_used_count u32              = 2\n",
      "llama_model_loader: - kv  11:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  12:                       llama.rope.freq_base f32              = 1000000.000000\n",
      "llama_model_loader: - kv  13:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  14:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  16:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  17:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  18:                      tokenizer.ggml.merges arr[str,58980]   = [\"▁ t\", \"i n\", \"e r\", \"▁ a\", \"h e...\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  20:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  21:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  22:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  23:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  24:                    tokenizer.chat_template str              = {{ bos_token }}{% for message in mess...\n",
      "llama_model_loader: - kv  25:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:   32 tensors\n",
      "llama_model_loader: - type q8_0:   64 tensors\n",
      "llama_model_loader: - type q4_K:  833 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 32768\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 8\n",
      "llm_load_print_meta: n_expert_used    = 2\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 1000000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 32768\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 46.70 B\n",
      "llm_load_print_meta: model size       = 24.62 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = ..\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.38 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  = 25216.25 MiB\n",
      "llm_load_tensors: offloading 0 repeating layers to GPU\n",
      "llm_load_tensors: offloaded 0/33 layers to GPU\n",
      "....................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 2048\n",
      "llama_new_context_with_model: freq_base  = 1000000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 1124/1124\n",
      "llama_new_context_with_model: compute buffer total size = 187.22 MiB\n",
      "AVX = 1 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 0 | AVX512_VNNI = 1 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n"
     ]
    }
   ],
   "source": [
    "# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.\n",
    "llm = Llama(\n",
    "  model_path=\"./mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf\",  # Download the model file first\n",
    "  n_ctx=2048,  # The max sequence length to use - note that longer sequence lengths require much more resources\n",
    "  n_threads=8,            # The number of CPU threads to use, tailor to your system and the resulting performance\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02222682-2d08-496c-8a68-f3bcbcc3b9f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =    2211.40 ms\n",
      "llama_print_timings:      sample time =      14.51 ms /    30 runs   (    0.48 ms per token,  2067.40 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2211.31 ms /     9 tokens (  245.70 ms per token,     4.07 tokens per second)\n",
      "llama_print_timings:        eval time =    7560.99 ms /    29 runs   (  260.72 ms per token,     3.84 tokens per second)\n",
      "llama_print_timings:       total time =    9868.26 ms\n"
     ]
    }
   ],
   "source": [
    "# Simple inference example\n",
    "output = llm(\n",
    "  \"[INST] hi [/INST]\", # Prompt\n",
    "  max_tokens=512,  # Generate up to 512 tokens\n",
    "  stop=[\"</s>\"],   # Example stop token - not necessarily correct for this specific model! Please check before using.\n",
    "  echo=True        # Whether to echo the prompt\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3ebd5e7-03ce-431c-8901-ae3d6535bf04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'cmpl-5b94a6eb-f4d4-47bb-9fab-0d1e98ce0d68',\n",
       " 'object': 'text_completion',\n",
       " 'created': 1703736342,\n",
       " 'model': './mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf',\n",
       " 'choices': [{'text': '[INST] hi [/INST] Hello! How can I help you today? If you have any questions about a specific topic or just want to chat, feel free to ask!',\n",
       "   'index': 0,\n",
       "   'logprobs': None,\n",
       "   'finish_reason': 'stop'}],\n",
       " 'usage': {'prompt_tokens': 9, 'completion_tokens': 29, 'total_tokens': 38}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a680f8a7-ee4a-4b23-ad5c-4c520ad6692b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
